{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "chars='BTSXPVE'\n",
    "\n",
    "graph = [[(1,5),('T','P')] , [(1,2),('S','X')], \\\n",
    "           [(3,5),('S','X')], [(6,),('E')], \\\n",
    "           [(3,2),('V','P')], [(4,5),('V','T')] ]\n",
    "\n",
    "\n",
    "def in_grammar(word):\n",
    "    if word[0] != 'B':\n",
    "        return False\n",
    "    node = 0    \n",
    "    for c in word[1:]:\n",
    "        transitions = graph[node]\n",
    "        try:\n",
    "            node = transitions[0][transitions[1].index(c)]\n",
    "        except ValueError: # using exceptions for flow control in python is common\n",
    "            return False\n",
    "    return True        \n",
    "      \n",
    "def sequenceToWord(sequence):\n",
    "    \"\"\"\n",
    "    converts a sequence (one-hot) in a reber string\n",
    "    \"\"\"\n",
    "    reberString = ''\n",
    "    for s in sequence:\n",
    "        index = np.where(s==1.)[0][0]\n",
    "        reberString += chars[index]\n",
    "    return reberString\n",
    "    \n",
    "def generateSequences(minLength):\n",
    "    while True:\n",
    "        inchars = ['B']\n",
    "        node = 0\n",
    "        outchars = []    \n",
    "        while node != 6:\n",
    "            transitions = graph[node]\n",
    "            i = np.random.randint(0, len(transitions[0]))\n",
    "            inchars.append(transitions[1][i])\n",
    "            outchars.append(transitions[1])\n",
    "            node = transitions[0][i]\n",
    "        if len(inchars) > minLength:  \n",
    "            return inchars, outchars\n",
    "\n",
    "\n",
    "def get_one_example(minLength):\n",
    "    inchars, outchars = generateSequences(minLength)\n",
    "    inseq = []\n",
    "    outseq= []\n",
    "    for i,o in zip(inchars, outchars): \n",
    "        inpt = np.zeros(7)\n",
    "        inpt[chars.find(i)] = 1.     \n",
    "        outpt = np.zeros(7)\n",
    "        for oo in o:\n",
    "            outpt[chars.find(oo)] = 1.\n",
    "        inseq.append(inpt)\n",
    "        outseq.append(outpt)\n",
    "    return inseq, outseq\n",
    "\n",
    "\n",
    "def get_char_one_hot(char):\n",
    "    char_oh = np.zeros(7)\n",
    "    for c in char:\n",
    "        char_oh[chars.find(c)] = 1.\n",
    "    return [char_oh] \n",
    "    \n",
    "def get_n_examples(n, minLength=10):\n",
    "    examples = []\n",
    "    for i in xrange(n):\n",
    "        examples.append(get_one_example(minLength))\n",
    "    return examples\n",
    "\n",
    "emb_chars = \"TP\"\n",
    "\n",
    "\n",
    "def get_one_embedded_example(minLength=10):\n",
    "    i, o = get_one_example(minLength)\n",
    "    emb_char = emb_chars[np.random.randint(0, len(emb_chars))]\n",
    "    new_in = get_char_one_hot(('B',))\n",
    "    new_in += get_char_one_hot((emb_char,))\n",
    "    new_out= get_char_one_hot(emb_chars)\n",
    "    new_out+= get_char_one_hot('B',)\n",
    "    new_in += i\n",
    "    new_out += o\n",
    "    new_in += get_char_one_hot(('E',))\n",
    "    new_in += get_char_one_hot((emb_char,))\n",
    "    new_out += get_char_one_hot((emb_char, ))\n",
    "    new_out += get_char_one_hot(('E',))\n",
    "    return new_in, new_out\n",
    "    \n",
    "def get_n_embedded_examples(n, minLength=10):\n",
    "    examples = []\n",
    "    for i in xrange(n):\n",
    "        examples.append(get_one_embedded_example(minLength))\n",
    "    return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "\n",
    "dtype=theano.config.floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# squashing of the gates should result in values between 0 and 1\n",
    "# therefore we use the logistic function\n",
    "sigma = lambda x: 1 / (1 + T.exp(-x))\n",
    "\n",
    "\n",
    "# for the other activation function we use the tanh\n",
    "act = T.tanh\n",
    "\n",
    "# sequences: x_t\n",
    "# prior results: h_tm1, c_tm1\n",
    "# non-sequences: W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xy, W_hy, W_cy, b_y\n",
    "def one_lstm_step(x_t, h_tm1, c_tm1, W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xy, W_ho, W_cy, b_o, W_hy, b_y):\n",
    "    i_t = sigma(theano.dot(x_t, W_xi) + theano.dot(h_tm1, W_hi) + theano.dot(c_tm1, W_ci) + b_i)\n",
    "    f_t = sigma(theano.dot(x_t, W_xf) + theano.dot(h_tm1, W_hf) + theano.dot(c_tm1, W_cf) + b_f)\n",
    "    c_t = f_t * c_tm1 + i_t * act(theano.dot(x_t, W_xc) + theano.dot(h_tm1, W_hc) + b_c) \n",
    "    o_t = sigma(theano.dot(x_t, W_xo)+ theano.dot(h_tm1, W_ho) + theano.dot(c_t, W_co)  + b_o)\n",
    "    h_t = o_t * act(c_t)\n",
    "    y_t = sigma(theano.dot(h_t, W_hy) + b_y) \n",
    "    return [h_t, c_t, y_t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#TODO: Use a more appropriate initialization method\n",
    "def sample_weights(sizeX, sizeY):\n",
    "    values = np.ndarray([sizeX, sizeY], dtype=dtype)\n",
    "    for dx in xrange(sizeX):\n",
    "        vals = np.random.uniform(low=-1., high=1.,  size=(sizeY,))\n",
    "        #vals_norm = np.sqrt((vals**2).sum())\n",
    "        #vals = vals / vals_norm\n",
    "        values[dx,:] = vals\n",
    "    _,svs,_ = np.linalg.svd(values)\n",
    "    #svs[0] is the largest singular value                      \n",
    "    values = values / svs[0]\n",
    "    return values  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_in = 7 # for embedded reber grammar\n",
    "n_hidden = n_i = n_c = n_o = n_f = 100\n",
    "n_y = 7 # for embedded reber grammar\n",
    "\n",
    "# initialize weights\n",
    "# i_t and o_t should be \"open\" or \"closed\"\n",
    "# f_t should be \"open\" (don't forget at the beginning of training)\n",
    "# we try to archive this by appropriate initialization of the corresponding biases \n",
    "\n",
    "W_xi = theano.shared(sample_weights(n_in, n_i))  \n",
    "W_hi = theano.shared(sample_weights(n_hidden, n_i))  \n",
    "W_ci = theano.shared(sample_weights(n_c, n_i))  \n",
    "b_i = theano.shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_i)))\n",
    "W_xf = theano.shared(sample_weights(n_in, n_f)) \n",
    "W_hf = theano.shared(sample_weights(n_hidden, n_f))\n",
    "W_cf = theano.shared(sample_weights(n_c, n_f))\n",
    "b_f = theano.shared(np.cast[dtype](np.random.uniform(0, 1.,size = n_f)))\n",
    "W_xc = theano.shared(sample_weights(n_in, n_c))  \n",
    "W_hc = theano.shared(sample_weights(n_hidden, n_c))\n",
    "b_c = theano.shared(np.zeros(n_c, dtype=dtype))\n",
    "W_xo = theano.shared(sample_weights(n_in, n_o))\n",
    "W_ho = theano.shared(sample_weights(n_hidden, n_o))\n",
    "W_co = theano.shared(sample_weights(n_c, n_o))\n",
    "b_o = theano.shared(np.cast[dtype](np.random.uniform(-0.5,.5,size = n_o)))\n",
    "W_hy = theano.shared(sample_weights(n_hidden, n_y))\n",
    "b_y = theano.shared(np.zeros(n_y, dtype=dtype))\n",
    "\n",
    "c0 = theano.shared(np.zeros(n_hidden, dtype=dtype))\n",
    "h0 = T.tanh(c0)\n",
    "\n",
    "params = [W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xo, W_ho, W_co, b_o, W_hy, b_y, c0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#first dimension is time\n",
    "\n",
    "#input \n",
    "v = T.matrix(dtype=dtype)\n",
    "\n",
    "# target\n",
    "target = T.matrix(dtype=dtype)\n",
    "\n",
    "# hidden and outputs of the entire sequence\n",
    "[h_vals, _, y_vals], _ = theano.scan(fn=one_lstm_step, \n",
    "                                  sequences = dict(input=v, taps=[0]), \n",
    "                                  outputs_info = [h0, c0, None ], # corresponds to return type of fn\n",
    "\n",
    "                                     \n",
    "                                     non_sequences = [W_xi, W_hi, W_ci, b_i, W_xf, W_hf, W_cf, b_f, W_xc, W_hc, b_c, W_xo, W_ho, W_co, b_o, W_hy, b_y] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print \"this step may take time: compute all gradients relative to the cost\"\n",
    "\n",
    "cost = -T.mean(target * T.log(y_vals)+ (1.- target) * T.log(1. - y_vals))\n",
    "\n",
    "# learning rate\n",
    "lr = np.cast[dtype](.1)\n",
    "learning_rate = theano.shared(lr)\n",
    "\n",
    "print \"numparams: \" + str(len(params))\n",
    "\n",
    "gparams = []\n",
    "for param in params:\n",
    "    gparam = T.grad(cost, param)\n",
    "    gparams.append(gparam)\n",
    "\n",
    "    \n",
    "print \"appending updates\"\n",
    "updates=[]\n",
    "for param, gparam in zip(params, gparams):\n",
    "    updates.append((param, param - gparam * learning_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_data = get_n_embedded_examples(1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "learn_rnn_fn = theano.function(inputs = [v, target],\n",
    "                               outputs = cost,\n",
    "                               updates = updates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nb_epochs=250\n",
    "train_errors = np.ndarray(nb_epochs)\n",
    "def train_rnn(train_data):      \n",
    "    for x in range(nb_epochs):\n",
    "        error = 0.\n",
    "        for j in range(len(train_data)):  \n",
    "            index = np.random.randint(0, len(train_data))\n",
    "            i, o = train_data[index]\n",
    "            train_cost = learn_rnn_fn(i, o)\n",
    "            #train_cost = train_f2(i, o)\n",
    "            error += train_cost\n",
    "        train_errors[x] = error \n",
    "        if x%10==0:\n",
    "            print \"epoch \"+str(x)+ \" error: \"+str(error)\n",
    "    \n",
    "    \n",
    "train_rnn(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(np.arange(nb_epochs), train_errors, 'b-')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('error')\n",
    "plt.ylim(0., 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = theano.function(inputs = [v], outputs = y_vals)\n",
    "\n",
    "test_data = get_n_embedded_examples(10)\n",
    "\n",
    "def print_out(test_data):\n",
    "    for i,o in test_data:\n",
    "        p = predictions(i)\n",
    "        print o[-2] # target\n",
    "        print np.asarray([0. if x!=np.argmax(p[-2]) else 1. for x in range(len(p[-2]))]) # prediction\n",
    "        print \n",
    "print_out(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
